<?php
define ('TYPE_IMAGE', '1');
define ('TYPE_MOVIE', '2');
define ('TYPE_RAR', '3');
define ('DATA_PATH', '/var/www/shadow.net/client');

require("queue.progress.php");   
require("../webnews/nntp.php");   

function Page_Load()
{
    global $argv;
    $messageKey = $argv[1];  
    set_time_limit(0);

    $messageDoc = QUEUE_DATA . "/{$messageKey}.pend";
    if (!file_exists($messageDoc)) return;
    echo "\n{$messageDoc}\n";
    $message = file_get_contents ($messageDoc);
    unlink ($messageDoc);
    echo $message;
    echo "\n\n";


    $unpacked    = simplexml_load_string($message);  
    $groupRenew  = "{$unpacked->renew}"; 
    $groupName   = "{$unpacked->groupname}"; 
    $groupFile   = "{$unpacked->location}"; 
    $groupCount  = "{$unpacked->count}"; 
    $groupStart  = "{$unpacked->start}"; 
    $groupParam  = "{$unpacked->param}"; 
    $groupDecode = "{$unpacked->decode}"; 
    $groupSaveas = "{$unpacked->saveas}"; 
    $groupEnd    = "{$unpacked->end}"; 



    $newsserver = new News_Usenet_Server;
    if (strlen($groupDecode) > 0) {
        $xml = $newsserver->Assemble ($groupDecode, $groupName, $messageKey);  
        file_put_contents ($groupSaveas, $xml);
        SetProgress($messageKey, 1, 1, 'COMPLETE', "");  
        return;     
    }
    else if (strlen($groupParam) > 0) {
        echo $newsserver->SearchNewsGroup ($groupName, $groupParam, $groupStart, $groupEnd, $messageKey);  
#        SetProgress($messageKey, 1, 1, 'COMPLETE', "");  
        return;     
    }
    else if (file_exists($groupFile)) {
        if ($groupRenew=='renew') { 
            unlink ($groupFile);
        }
        else {
            $temp = explode ('/', $groupFile);
            $name = array_pop($temp);
            $temp = implode ('/', $temp); 

            for ($fn=1;$fn<100;$fn++)
            {
                 $newname = "{$temp}/{$groupName},{$fn}.xml";
                 if (!file_exists($newname))
                 {
                     rename ($groupFile, $newname);
                     break;
                 }
            } 
        }
    }
    $database   = $newsserver->LoadNewsGroup($groupName, $groupStart, $groupCount, $messageKey);
    file_put_contents($groupFile, $database);
    echo filesize ($groupFile) . " bytes saved. \n";
    SetProgress($messageKey, 1, 1, 'COMPLETE', ""); 
} 
Page_Load ();

class News_Usenet_Server
{
    public $List;
    public $NNTP; 
    function __construct()
    {
        $info = array ('server'   => "news.usenet.net",
                        'username' => 'info@cyber8.net',
                        'password' => 'Password');
        $Server     = $info['server'];
        $Username   = $info['username'];
        $Password   = $info['password']; 
        $this->NNTP = new NNTP($Server, trim($Username), trim($Password));   
    }


    function LoadNewsGroup ($groupName, $groupStart, $numOfArticles, $messageKey, $Retry=0)
    { 
        $this->List = array();
        $nntp = $this->NNTP;
        if (!$nntp->connect())                                                
        {                                                
            header("Content-Error: " . $nntp->get_error_message());                                               
            echo "<b>".$messages_ini["error"]["nntp_fail"]."</b><br>";                                                
            echo $nntp->get_error_message().$nntp_server."<br>";                                                
            return;                                               
        }  
        
        if ( $gi=$nntp->join_group(trim($groupName)) )                                               
        {                                      
            $minimum    = $gi['start_id'];                               
            $maximum    = $gi['end_id'];                                 
            $start_id   = min ($minimum, $maximum);                        
            $max_id     = max ($minimum, $maximum);     
            if (strlen($groupStart) > 0) $max_id = $groupStart;
	    $end_id     = $max_id; //$start_id + $numOfArticles;  
	    $start_id   = $end_id - $numOfArticles;   
             
	    $end_id     = $start_id + $numOfArticles;  
            $nntp_query = "XOVER " . $start_id . "-" . $max_id;  
            $buf        = $nntp->send_request( $nntp_query );                                                   
            $response   = $nntp->parse_response($buf);         
            $cursor	= 0;
            $xml        = '';
            $groupData  = array('min'=>$start_id, 'max'=>$max_id); 
		   
echo $nntp_query . "\n=============================================\n";

print_r($response);
            if ($response["status"] == ARTICLE_OVERVIEW || $response["status"] == ARTICLE_HEAD)                                                
            {                                                             
                $buf = fgets($nntp->nntp, 4096);                    
                while (!preg_match("/^\.\s*$/", $buf))                                                
                {     
	            $art = new News_Usenet_Article($buf, 0, $groupName);
	
	            if (array_key_exists($art->regEx, $this->List))
	            {
	                if (array_key_exists($art->partKey, $this->List[$art->regEx]))
	                {
	                    $this->List[$art->regEx][$art->partKey][] = $art;
	                }
	                else
	                {
	                    $this->List[$art->regEx][$art->partKey] = array($art);
	                }
	            }
	            else
	            {
	                $this->List[$art->regEx] = array($art->partKey => array($art));
	            }                 
	                
                    $cursor ++;
                    if ($cursor >= $numOfArticles)  break;
                    if ($cursor % 1000 == 0) {
                        SetProgress($messageKey, $numOfArticles, $cursor, 'IN PROGRESS', "Retrieving {$groupName} information...") ; 
                    }
                    $buf = fgets($nntp->nntp, 4096);     
                }              
            }
        }    
               
        $nntp->quit();    

        if( sizeof ($this->List) < 5 && $Retry < 5) {
            echo sizeof ($this->List) . " articles. Retrying ({$Retry})...\n\n";
            return $this->LoadNewsGroup ($groupName, $groupStart - $numOfArticles, $numOfArticles, $messageKey, $Retry+1);

        }


        $this->List = array_reverse($this->List); 
        echo "\n\nCollating...\n";
        return $this->Collate ($groupName, $groupData);
    } 


    
    function Assemble ($file, $groupName, $messageKey)
    { 
	$fp   = fopen($file, 'r');   
	$size = filesize($file);   
                 
        echo "Download of {$size} bytes complete. Collating...\n"; 
        while (($buf = fgets($fp, 4096)) !== false)                                                
        {    
            $art = new News_Usenet_Article($buf, 0, $groupName);
	    if (array_key_exists($art->regEx, $this->List))
	    {
	        if (array_key_exists($art->partKey, $this->List[$art->regEx]))
	        {
	            $this->List[$art->regEx][$art->partKey][] = $art;
	        }
	        else
	        {
	            $this->List[$art->regEx][$art->partKey] = array($art);
	        }
	    }
	    else
	    {
	        $this->List[$art->regEx] = array($art->partKey => array($art));
	    }     
	    unset ($art);                 
        }              
        fclose ($fp);
        unlink ($file);
        return $this->Collate ($groupName);	
    }
    

    function Collate ($groupName, $groupData, $messageKey = NULL)
    {		
        $step = 500;
        $xml  = array();
        $xml[] = '<?xml version="1.0" encoding="ISO-8859-1"?>';
        $xml[] = '<list groupname="' . $groupName . '">';
        $cnt = 0;       
          
        while (list($name, $item) = each($groupData))
        {
            $xml[] = "<{$name}>{$item}</{$name}>";
        }          
        
        while (list($name, $item) = each($this->List))
        { 
            $box  = array(); 
            $post = array(); 
                   
            while (list($key, $cluster) = each($item))
            { 
                $ext     = array();
                $rar     = NULL;
                $first   = $cluster[0]; 
                $picture = preg_match ('/\.(jpg|jpeg|png|gif|bmp)/i', $first->Subject);
                $video   = preg_match ('/\.(m4v|flv|mpeg|mpg|mov|wmv|avi|mkv)/i', $first->Subject, $ext);
                $picture = $picture && !preg_match ('/(par2|RE\:)/i', $first->Subject);
                $video   = $video && !preg_match ('/(par2|RE\:|\.\d{3})/i', $first->Subject) && !$picture;
                if (!$video)
                {
                    $rar     = preg_match ('/\.(rar|r\d{2})/i', $first->Subject, $ext);
                    $rar     = $rar && !preg_match ('/(par2|RE\:|\.\d{3})/i', $first->Subject);
                }
                $picture = $picture && !$video && !$rar;
              
                if ($picture || $video || $rar)
                {
                    $post[] = $cluster;
                } 
            } 
         
            if (sizeof($post) > 0)
            {
                 $id    = $this->GetArticlesetId($post[0]);     
                 $ref   = $post[0][0]->Ref;              
                 $text  = $post[0][0]->Subject;        
                 $from  = $post[0][0]->From;
                 $date  = $post[0][0]->articleDate;
                 $enum  = $this->GetType ($text);
                 $type  = $enum == TYPE_IMAGE ? 'picture' : 'unknown';
                 $type  = $enum == TYPE_MOVIE ? 'wmv' : $type;
                 $type  = $enum == TYPE_RAR   ? 'rar' : $type;
                 if (preg_match ('/\.(m4v|flv|mpeg|mpg|mov|wmv|avi|mkv|mp4)/i', $text, $ext))
                    $type  = $ext[1] == 'm4v' || $ext[1] == 'mp4'   ? 'm4v' : $type;
                 $subj  =  iconv("ISO-8859-1", "UTF-8", $text); 
                 $xml[] = '    <article index="' . $cnt . '">';       
                 $xml[] = '        <id><![CDATA[' . $id . ']]></id>';   
                 $xml[] = '        <type>'.$type.'</type>';    
                 $xml[] = '        <name><![CDATA[' . htmlentities ($from['name']) . ']]></name>';  
                 $xml[] = '        <email><![CDATA[' . htmlentities ($from['email']) . ']]></email>'; 
                 $xml[] = '        <date><![CDATA[' . $date . ']]></date>';   
                 $xml[] = '        <subject><![CDATA[' . htmlentities ($subj) . ']]></subject>';  
                 $xml[] = '        <ref><![CDATA[' . htmlentities (utf8_encode( implode('|', $ref) )) . ']]></ref>';   
                 $xml[] = '        <items>';     
                 for ($i=0;$i<sizeof($post);$i++)
                 {
                     $id    = $this->GetArticlesetId($post[$i]);       
                     $text  = $post[$i][0]->Subject;
                     $subj  =  iconv("ISO-8859-1", "UTF-8", $text); 
	                 $xml[] = '            <article>';       
	                 $xml[] = '                <id><![CDATA[' . $id . ']]></id>';   
	                 $xml[] = '                <subject><![CDATA[' . htmlentities ($subj) . ']]></subject>';  
	                 $xml[] = '            </article>';       
                 }
                 $xml[] = '        </items>';      
                 $xml[] = '    </article>';     
                 $cnt ++;
             }   
        }  
        $xml[] = '</list>'; 
        
        return implode ("\n", $xml);        
    }    

    function GetArticlesetId ($cluster)
    {
        $ret = array();
        $srt = array();

        for ($i=0;$i<sizeof($cluster);$i++)
        { 
            if (sizeof($cluster) > 1 && preg_match('/\((\d+)\/(\d+)\)/', $cluster[$i]->Subject, $matches))
            {
                $num = $matches[1];
                $cnt = $matches[2]; 
                $ret[$num - 1] = $cluster[$i]->messageId; 
            }
            else
            { 
                $ret[] = $cluster[$i]->messageId;
            }
        }
        $value = array();
        for ($i=0;$i<sizeof($ret);$i++) 
        {
            if (isset($ret[$i]))
            {
                $value[] = $ret[$i];
            }
        } 
        return implode('-', $value);
    }            

    function GetType ($str)
    {
        $picture = preg_match ('/\.(jpg|jpeg|png|gif|bmp)/i', $str);
        $video   = preg_match ('/\.(m4v|flv|mpeg|mpg|mov|wmv|avi|mkv)/i', $str); 
        if (!$video)
        {
            $rar = preg_match ('/\.(rar|r\d{2})/i', $str); 
            if ($rar)
            {
                return TYPE_RAR;
            } 
        }
        return $video && !$picture ? TYPE_MOVIE : TYPE_IMAGE;
    }


    function SearchNewsGroup ($groupName, $param = NULL, $start = NULL, $end = NULL, $messageKey = NULL)
    {
	echo $param . ',' . $start . ',' . $end;
	$nntp = $this->NNTP;
 
        $file = DATA_PATH . "/{$messageKey}.dat"; 
	$fp   = fopen ($file, 'a');
		
        if (!$nntp->connect())                                                
        {                                                
            header("Content-Error: " . $nntp->get_error_message());                                               
            echo "<b>".$messages_ini["error"]["nntp_fail"]."</b><br>";                                                
            echo $nntp->get_error_message().$nntp_server."<br>";                                                
            return;                                               
        }   
		
        echo "Searching group {$groupName} for '$param'\n"; 
        if ( $gi=$nntp->join_group(trim($groupName)) )                                               
        {
            if ($param) $param = strtolower($param);                                     
            $minimum    = $gi['start_id'];                               
            $maximum    = $gi['end_id'];                                 
            $start_id   = $start ? $start : min ($minimum, $maximum);                        
            $max_id     = $end   ? $end   : max ($minimum, $maximum);  
            $size       = $max_id - $start_id;
	    $count      = 0;
	    $span       = floor ($size / 25);
            $nntp_query = "XOVER {$start_id}-{$max_id}";    
            $buf        = $nntp->send_request( $nntp_query );                                                   
            $response   = $nntp->parse_response($buf);   
            $alike      = 0;
            echo "Searching $size articles\n"; 
	    echo "$nntp_query\n"; 
	    echo "$buf\n"; 

            SetProgress($messageKey, $size, $count, 'IN PROGRESS', "Connected to {$groupName}. Waiting for response...") ; 

            if ($response["status"] == ARTICLE_OVERVIEW || $response["status"] == ARTICLE_HEAD)                                                
            {                                         
                $buf = fgets($nntp->nntp, 4096);    
                while (!preg_match("/^\.\s*$/", $buf))                                                
                {
                    if ($param == NULL || strpos(strtolower($buf), $param)!==false)
		    {
                        fputs($fp, $buf); 
                        $alike ++;
                        if ($alike > ARTICLE_MAX) break;
                    }   
                    $count ++;
                    if ($count % 25000 == 0)
                    {
                        $sz = filesize($file); 
                        SetProgress ($messageKey, $size, $count, 'IN PROGRESS', 
                           "Found $alike articles like $param in $start_id thru $max_id...");
                    } 
                    $buf = fgets($nntp->nntp, 4096);     
                }   
                SetProgress ($messageKey, $size, $count, 'IN PROGRESS', "Retrieving $sz bytes $start_id $max_id...");
            }
        }          
        SetProgress($messageKey, $size, $size, 'COMPLETE', 
                        "Found $alike articles like $param in $start_id thru $max_id...");              
        $nntp->quit();     
	fclose ($fp); 
	unset ($fp);
	return $file; 	
    }  
}




class News_Usenet_Article
{
    var $messageId;
    var $Subject;
    var $From;
    var $articleDate;
    var $articleBytes;
    var $articleLines;
    var $Ref;
    var $regEx;
    var $articleList;
    var $partList;
    var $Level;
    var $prefix;
    var $matchKey;
    var $partKey;
    var $groupName;
    function News_Usenet_Article($buf, $level=1, $group='')
    {
        $this->groupName    = $group;
        $this->articleList  = array();
        $this->partList     = array();
        
        $elements           = preg_split("/\t/", $buf);                                            
        $this->messageId    = $elements [0];                                       
        $this->Subject      = $elements [1];                                                 
        $this->From         = decode_sender($elements[2]);                                            
        $this->articleDate  = $elements [3];                                        
        $this->articleBytes = $elements [6];                                                
        $this->articleLines = $elements [7];                                               
        $this->regEx        = $this->_getRegEx();
        $this->partKey      = $this->_getPartKey();
        
        $arr = preg_split("/\s+/", trim($elements [8]));
        $this->Ref = array();
        for ($i=0;$i<sizeof($arr);$i++)
        {
            $sp = explode (':', $arr[$i]);
            if (sizeof($sp) > 1)
            {
                $this->Ref[] = $sp[0];
            }
        } 
    }
    
    function _getRegEx()
    { 
        $regex        = preg_replace('/["\-\/\+\[\]\(\)\*]/', '.', $this->Subject);
        $regex        = preg_replace('/(\d+)/', '\d+', $regex);
        $regex        = preg_replace('/(\s+)/', '\s+', $regex); 
        
        if (preg_match('/\.(jpg|png|gif|bmp|wmv|rar|r\d{2})/i', $regex, $arr))
        {
            $regex = substr($regex, 0, strpos($regex, $arr[1]));
        }
        return $regex;
    }
    
    function _getPartKey()
    {
        $reg = '/(.*)\((\d+)\/(\d+)\)/';
        $key = $this->Subject;  
        if (preg_match($reg, $key, $info))
        {
            $key = $info[1]; 
        }
        
        return $key;
    }    
    
    function _getPrefix($buf)
    {
        $name = $this->_getSubject($buf);
        $pos   = strpos($name, '[')!==false;
        if ($pos)
        {
            return substr($name, 0, $pos);
        }
        return '';
    }
    
    function _getSubject($buf)
    {
        $elements = preg_split("/\t/", $buf);                            
        return $elements [1];         
    }    
    
    function _Parse($subject)
    {             
        if (preg_match('/(.*)\((\d+)\/(\d+)\)/', $subject, $info))
        {
            return $info[1];
        } 
        return NULL;
    } 
    
    function Match ($buf)
    {   
        return preg_match ('/' . $this->regEx . '/', $this->_getSubject($buf), $array); 
    }
    
    function AddPart($article, $index)
    {
        $this->partList[$index - 1] = $article;
    }
    
    function Add($buf, $keymatch=0)
    {
        $reg = '/(.*)\((\d+)\/(\d+)\)/';
        $key = $this->_getSubject($buf); 
        $tmp = new News_Usenet_Article($buf);
        $idx = -1;
        $sz  = 1;
        
        if (preg_match($reg, $key, $info))
        {
            $key = $info[1];
            $idx = $info[2];
            $sz  = $info[3]; 
        } 
        if (array_key_exists($this->regEx, $this->articleList))
        {
            $article = $this->articleList[$this->regEx];
            if (array_key_exists($key, $article))
            {
                $this->articleList[$this->regEx][$key][] = $tmp;
            }
            else 
            {
                $this->articleList[$this->regEx][$key] = array($tmp);
            }
        }
        else
        {
            $this->articleList[$this->regEx] = array($key=>array($tmp));
        }                      
    }
    
    function _getKey()
    {
        $ret = array();
        $ret[] = $this->messageId;
        if (sizeof($this->articleList) > 0)
        { 
	        foreach ($this->articleList as $item)
	        {
	            if (is_array($item))
	            {
	                foreach ($item as $i) 
	                    foreach ($i as $x)
	                        $ret[] = $x->_getKey(); 
	            }
	            else $ret[] = $item->_getKey();
	        } 
        } 
        return implode('-', $ret);
    } 
}
